<!DOCTYPE html>
<html lang="en">

<!-- Head tag -->
<head><meta name="generator" content="Hexo 3.9.0">

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">

    <!--Description-->
    
        <meta name="description" content="12345678910111213141516171819202122232425262728293031323334353637我们可以用一个settings.py做个简单的介绍和解析：例： ----&amp;gt;# -*- coding: utf-8 -*-# Scrapy settings for ">
    

    <!--Author-->
    
        <meta name="author" content="ck">
    

    <!--Open Graph Title-->
    
        <meta property="og:title" content="scrapy爬虫框架配置--settings">
    

    <!--Open Graph Description-->
    

    <!--Open Graph Site Name-->
    <meta property="og:site_name" content="CK&#39;s blogs">

    <!--Type page-->
    
        <meta property="og:type" content="article">
    

    <!--Page Cover-->
    

    <meta name="twitter:card" content="summary">
    

    <!-- Title -->
    
    <title>scrapy爬虫框架配置--settings - CK&#39;s blogs</title>

    <!-- Bootstrap Core CSS -->
    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.2/css/bootstrap.min.css" integrity="sha384-y3tfxAZXuh4HwSYylfB+J125MxIs6mR5FOHamPBG064zB+AFeWH94NdvaCBm8qnd" crossorigin="anonymous">

    <!-- Custom Fonts -->
    <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" type="text/css">

    <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
        <script src="//oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
        <script src="//oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
    <![endif]-->

    <!-- Gallery -->
    <link href="//cdnjs.cloudflare.com/ajax/libs/featherlight/1.3.5/featherlight.min.css" type="text/css" rel="stylesheet">

    <!-- Custom CSS -->
    <link rel="stylesheet" href="/blog/css/style.css">

    <!-- Google Analytics -->
    


</head>


<body>

<div class="bg-gradient"></div>
<div class="bg-pattern"></div>

<!-- Menu -->
<!--Menu Links and Overlay-->
<div class="menu-bg">
    <div class="menu-container">
        <ul>
            
            <li class="menu-item">
                <a href="/blog/">
                    Home
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/archives">
                    Archives
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/2019/09/04/本人简历/">
                    About
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/tags">
                    Tags
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/categories">
                    Categories
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/blog/contact.html">
                    Contact
                </a>
            </li>
            
        </ul>
    </div>
</div>

<!--Hamburger Icon-->
<nav>
    <a href="#menu"></a>
</nav>

<div class="container">

    <!-- Main Content -->
    <div class="row">
    <div class="col-sm-12">

        <!--Title and Logo-->
        <header>
    <div class="logo">
        <a href="/blog/"><i class="logo-icon fa fa-cube" aria-hidden="true"></i></a>
        
    </div>
</header>

        <section class="main">
            
<div class="post">

    <div class="post-header">
        <h1 class="title">
            <a href="/blog/2019/01/18/scrapy爬虫框架配置-settings/">
                scrapy爬虫框架配置--settings
            </a>
        </h1>
        <div class="post-info">
            
                <span class="date">2019-01-18</span>
            
            
            
        </div>
    </div>

    <div class="content">

        <!-- Gallery -->
        

        <!-- Post Content -->
        <figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line">我们可以用一个settings.py做个简单的介绍和解析：</span><br><span class="line">例： ----&gt;</span><br><span class="line"></span><br><span class="line"># -*- coding: utf-8 -*-</span><br><span class="line"></span><br><span class="line"># Scrapy settings for xigua project</span><br><span class="line">#</span><br><span class="line"># For simplicity, this file contains only settings considered important or</span><br><span class="line"># commonly used. You can find more settings consulting the documentation:</span><br><span class="line">#</span><br><span class="line">#     https://doc.scrapy.org/en/latest/topics/settings.html</span><br><span class="line">#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html</span><br><span class="line">#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html</span><br><span class="line"></span><br><span class="line">BOT_NAME = &apos;xigua&apos;    # Scrapy项目的名字,这将用来构造默认 User-Agent,同时也用来log,当您使用 startproject 命令创建项目时其也被自动赋值。</span><br><span class="line"></span><br><span class="line">SPIDER_MODULES = [&apos;xigua.spiders&apos;]　　#Scrapy搜索spider的模块列表 默认: [xxx.spiders]</span><br><span class="line">NEWSPIDER_MODULE = &apos;xigua.spiders&apos;　　#使用 genspider 命令创建新spider的模块。默认: &apos;xxx.spiders&apos;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># Crawl responsibly by identifying yourself (and your website) on the user-agent</span><br><span class="line">#爬取的默认User-Agent，除非被覆盖</span><br><span class="line">#USER_AGENT = &apos;xigua (+http://www.yourdomain.com)&apos;</span><br><span class="line"></span><br><span class="line"># Obey robots.txt rules</span><br><span class="line">#如果启用,Scrapy将会采用 robots.txt策略，常使用不遵循Flase</span><br><span class="line">ROBOTSTXT_OBEY = False</span><br><span class="line"></span><br><span class="line"># Configure maximum concurrent requests performed by Scrapy (default: 16)</span><br><span class="line">#Scrapy downloader 并发请求(concurrent requests)的最大值,默认: 16</span><br><span class="line">#CONCURRENT_REQUESTS = 32</span><br><span class="line"></span><br><span class="line"># Configure a delay for requests for the same website (default: 0)</span><br><span class="line">#未同意网站的请求配置延迟（默认为0）</span><br><span class="line"># See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay</span><br><span class="line"># See also autothrottle settings and docs</span><br><span class="line">DOWNLOAD_DELAY = 3　　# 下载器延迟时间，下载器在同一网站面前需要等待的时间，该选项可以用来限制爬取速度,减轻服务器压力。同时也支持小数:0.25 以秒为单# The download delay setting will honor only one of:</span><br></pre></td></tr></table></figure>

<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br></pre></td><td class="code"><pre><span class="line"># 下载延迟设置，只能有一个生效</span><br><span class="line">#CONCURRENT_REQUESTS_PER_DOMAIN = 16　　# 对单个网站进行并发请求的最大值</span><br><span class="line">#CONCURRENT_REQUESTS_PER_IP = 16　　</span><br><span class="line">#对单个ip进行并发请求的最大值，如果非0，则忽略，CONCURRENT_REQUESTS_PER_DOMAIN 设定,使用该设定。 </span><br><span class="line">#也就是说,并发限制将针对IP,而不是网站。该设定也影响 DOWNLOAD_DELAY: 如果 CONCURRENT_REQUESTS_PER_IP 非0,下载延迟应用在IP而不是网站上。</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># Disable cookies (enabled by default)</span><br><span class="line"># 禁用cookie（默认情况下启用）</span><br><span class="line">#COOKIES_ENABLED = False</span><br><span class="line"></span><br><span class="line"># Disable Telnet Console (enabled by default)</span><br><span class="line"># 禁用Telent控制台（默认启用）</span><br><span class="line">#TELNETCONSOLE_ENABLED = False</span><br><span class="line"></span><br><span class="line"># Override the default request headers:</span><br><span class="line"># 覆盖默认请求标头，也可以加入请求头，获取同样来自开发着工具，</span><br><span class="line"># 很多网站都会检查客户端的headers，比如豆瓣就是每一个请求都检查headers的user_agent，否则只会返回403，可以开启 USER_AGENT 请求头</span><br><span class="line">#DEFAULT_REQUEST_HEADERS = &#123;</span><br><span class="line">#   &apos;Accept&apos;: &apos;text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8&apos;,</span><br><span class="line">#   &apos;Accept-Language&apos;: &apos;en&apos;,</span><br><span class="line">#&#125;</span><br><span class="line"></span><br><span class="line"># Enable or disable spider middlewares</span><br><span class="line"># 启用或禁用蜘蛛中间件</span><br><span class="line"># See https://doc.scrapy.org/en/latest/topics/spider-middleware.html</span><br><span class="line">#SPIDER_MIDDLEWARES = &#123;</span><br><span class="line">#    &apos;xigua.middlewares.XiguaSpiderMiddleware&apos;: 543,</span><br><span class="line">#&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># Enable or disable downloader middlewares</span><br><span class="line"># 启用或禁用下载器中间件</span><br><span class="line"># See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html</span><br><span class="line">#DOWNLOADER_MIDDLEWARES = &#123;</span><br><span class="line">#    &apos;xigua.middlewares.XiguaDownloaderMiddleware&apos;: 543,</span><br><span class="line">#&#125;</span><br><span class="line"></span><br><span class="line"># Enable or disable extensions</span><br><span class="line">#启用或禁用扩展程序</span><br><span class="line"># See https://doc.scrapy.org/en/latest/topics/extensions.html</span><br><span class="line">#EXTENSIONS = &#123;</span><br><span class="line">#    &apos;scrapy.extensions.telnet.TelnetConsole&apos;: None,</span><br><span class="line">#&#125;</span><br><span class="line"></span><br><span class="line"># Configure item pipelines</span><br><span class="line"># 配置项目管道，如下载图片的图片管道，分布式爬虫多爬虫的pipeline，结尾int值是优先级，可以理解为权重，以逗号间隔，是个集合</span><br><span class="line"># See https://doc.scrapy.org/en/latest/topics/item-pipeline.html</span><br><span class="line">ITEM_PIPELINES = &#123;</span><br><span class="line">   &apos;xigua.pipelines.XiguaPipeline&apos;: 300,</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"># Enable and configure the AutoThrottle extension (disabled by default)</span><br><span class="line"># 启用或配置AutoThrottle扩展（默认情况下禁用）</span><br><span class="line"># See https://doc.scrapy.org/en/latest/topics/autothrottle.html</span><br><span class="line">#AUTOTHROTTLE_ENABLED = True</span><br><span class="line"></span><br><span class="line"># The initial download delay</span><br><span class="line"># 初始下载延迟</span><br><span class="line">#AUTOTHROTTLE_START_DELAY = 5</span><br><span class="line"></span><br><span class="line"># The maximum download delay to be set in case of high latencies</span><br><span class="line"># 在高延迟的情况下设置最大下载延迟</span><br><span class="line">#AUTOTHROTTLE_MAX_DELAY = 60</span><br><span class="line"></span><br><span class="line"># The average number of requests Scrapy should be sending in parallel to each remote server</span><br><span class="line"># Scrapy请求的平均数量应该并行发送每个远程服务器</span><br><span class="line">#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0</span><br></pre></td></tr></table></figure>

<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br></pre></td><td class="code"><pre><span class="line"># Enable showing throttling stats for every response received:</span><br><span class="line"># 启用显示所收到的每个响应的调节统计信息</span><br><span class="line">#AUTOTHROTTLE_DEBUG = False</span><br><span class="line"></span><br><span class="line"># Enable and configure HTTP caching (disabled by default)</span><br><span class="line"># 启用或配置 Http 缓存（默认情况下禁用）</span><br><span class="line"># See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings</span><br><span class="line">#HTTPCACHE_ENABLED = True</span><br><span class="line">#HTTPCACHE_EXPIRATION_SECS = 0</span><br><span class="line">#HTTPCACHE_DIR = &apos;httpcache&apos;</span><br><span class="line">#HTTPCACHE_IGNORE_HTTP_CODES = []</span><br><span class="line">#HTTPCACHE_STORAGE = &apos;scrapy.extensions.httpcache.FilesystemCacheStorage&apos;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">这就是一个完整的setting.py的接本情况：</span><br><span class="line">接下来再来详细看看，并且还有哪些可以配置的呢？</span><br><span class="line">-----》</span><br><span class="line"></span><br><span class="line">#-----------------------日志文件配置-----------------------------------</span><br><span class="line"># 默认: True,是否启用logging。</span><br><span class="line"># LOG_ENABLED=True</span><br><span class="line"># 默认: &apos;utf-8&apos;,logging使用的编码。</span><br><span class="line"># LOG_ENCODING=&apos;utf-8&apos;</span><br><span class="line"># 它是利用它的日志信息可以被格式化的字符串。默认值：&apos;%(asctime)s [%(name)s] %(levelname)s: %(message)s&apos;</span><br><span class="line"># LOG_FORMAT=&apos;%(asctime)s [%(name)s] %(levelname)s: %(message)s&apos;</span><br><span class="line"># 它是利用它的日期/时间可以格式化字符串。默认值： &apos;%Y-%m-%d %H:%M:%S&apos;</span><br><span class="line"># LOG_DATEFORMAT=&apos;%Y-%m-%d %H:%M:%S&apos;</span><br><span class="line">#日志文件名</span><br><span class="line">#LOG_FILE = &quot;dg.log&quot;</span><br><span class="line">#日志文件级别,默认值：“DEBUG”,log的最低级别。可选的级别有: CRITICAL、 ERROR、WARNING、INFO、DEBUG 。</span><br><span class="line">LOG_LEVEL = &apos;WARNING&apos;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># -----------------------------robots协议---------------------------------------------</span><br><span class="line"># Obey robots.txt rules</span><br><span class="line"># robots.txt 是遵循 Robot协议 的一个文件，它保存在网站的服务器中，它的作用是，告诉搜索引擎爬虫，</span><br><span class="line"># 本网站哪些目录下的网页 不希望 你进行爬取收录。在Scrapy启动后，会在第一时间访问网站的 robots.txt 文件，</span><br><span class="line"># 然后决定该网站的爬取范围。</span><br><span class="line"># ROBOTSTXT_OBEY = True</span><br><span class="line"></span><br><span class="line"># 对于失败的HTTP请求(如超时)进行重试会降低爬取效率，当爬取目标基数很大时，舍弃部分数据不影响大局，提高效率</span><br><span class="line">RETRY_ENABLED = False</span><br><span class="line">#请求下载超时时间，默认180秒</span><br><span class="line">DOWNLOAD_TIMEOUT=20</span><br><span class="line"># 这是响应的下载器下载的最大尺寸，默认值：1073741824 (1024MB)</span><br><span class="line"># DOWNLOAD_MAXSIZE=1073741824</span><br><span class="line"># 它定义为响应下载警告的大小，默认值：33554432 (32MB)</span><br><span class="line"># DOWNLOAD_WARNSIZE=33554432</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># ------------------------全局并发数的一些配置:-------------------------------</span><br><span class="line"># Configure maximum concurrent requests performed by Scrapy (default: 16)</span><br><span class="line"># 默认 Request 并发数：16</span><br><span class="line"># CONCURRENT_REQUESTS = 32</span><br><span class="line"># 默认 Item 并发数：100</span><br><span class="line"># CONCURRENT_ITEMS = 100</span><br><span class="line"># The download delay setting will honor only one of:</span><br><span class="line"># 默认每个域名的并发数：8</span><br><span class="line">#CONCURRENT_REQUESTS_PER_DOMAIN = 16</span><br><span class="line"># 每个IP的最大并发数：0表示忽略</span><br><span class="line"># CONCURRENT_REQUESTS_PER_IP = 0</span><br><span class="line"></span><br><span class="line"># Configure a delay for requests for the same website (default: 0)</span><br><span class="line"># See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay</span><br><span class="line"># See also autothrottle settings and docs</span><br><span class="line">#DOWNLOAD_DELAY 会影响 CONCURRENT_REQUESTS，不能使并发显现出来,设置下载延迟</span><br><span class="line">#DOWNLOAD_DELAY = 3</span><br><span class="line"></span><br><span class="line"># Disable cookies (enabled by default)</span><br><span class="line">#禁用cookies,有些站点会从cookies中判断是否为爬虫</span><br><span class="line"># COOKIES_ENABLED = True</span><br><span class="line"># COOKIES_DEBUG = True</span><br><span class="line"></span><br><span class="line"># Crawl responsibly by identifying yourself (and your website) on the user-agent</span><br><span class="line"># 它定义了在抓取网站所使用的用户代理，默认值：“Scrapy / VERSION“</span><br><span class="line">#USER_AGENT = &apos; (+http://www.yourdomain.com)&apos;</span><br><span class="line"></span><br><span class="line"># Override the default request headers:</span><br><span class="line">DEFAULT_REQUEST_HEADERS = &#123;</span><br><span class="line">  &apos;Accept&apos;: &apos;text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8&apos;,</span><br><span class="line">  &apos;Accept-Language&apos;: &apos;en&apos;,</span><br><span class="line">&#125;</span><br><span class="line"># Enable or disable spider middlewares</span><br><span class="line"># See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html</span><br><span class="line">SPIDER_MIDDLEWARES = &#123;</span><br><span class="line">    &apos;companyNews.middlewares.UserAgentmiddleware&apos;: 401,</span><br><span class="line">    &apos;companyNews.middlewares.ProxyMiddleware&apos;:426,</span><br><span class="line">&#125;</span><br><span class="line"># Enable or disable downloader middlewares</span><br><span class="line"># See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html</span><br><span class="line">DOWNLOADER_MIDDLEWARES = &#123;</span><br><span class="line">    &apos;companyNews.middlewares.UserAgentmiddleware&apos;: 400,</span><br><span class="line">    &apos;companyNews.middlewares.ProxyMiddleware&apos;:425,</span><br><span class="line">    # &apos;scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware&apos;:423,</span><br><span class="line">     # &apos;companyNews.middlewares.CookieMiddleware&apos;: 700,</span><br><span class="line">&#125;</span><br><span class="line">MYEXT_ENABLED=True      # 开启扩展</span><br><span class="line">IDLE_NUMBER=12           # 配置空闲持续时间单位为 360个 ，一个时间单位为5s</span><br><span class="line"># Enable or disable extensions</span><br><span class="line"># See http://scrapy.readthedocs.org/en/latest/topics/extensions.html</span><br><span class="line"># 在 EXTENSIONS 配置，激活扩展</span><br><span class="line">EXTENSIONS = &#123;</span><br><span class="line">    # &apos;scrapy.extensions.telnet.TelnetConsole&apos;: None,</span><br><span class="line">    &apos;companyNews.extensions.RedisSpiderSmartIdleClosedExensions&apos;: 500,</span><br><span class="line">&#125;</span><br><span class="line"># Configure item pipelines</span><br><span class="line"># See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html</span><br><span class="line"># 注意:自定义pipeline的优先级需高于Redispipeline,因为RedisPipeline不会返回item,</span><br><span class="line"># 所以如果RedisPipeline优先级高于自定义pipeline,那么自定义pipeline无法获取到item</span><br><span class="line">ITEM_PIPELINES = &#123;</span><br><span class="line">     #将清除的项目在redis进行处理，# 将RedisPipeline注册到pipeline组件中(这样才能将数据存入Redis)</span><br><span class="line">    # &apos;scrapy_redis.pipelines.RedisPipeline&apos;: 400,</span><br><span class="line">    &apos;companyNews.pipelines.companyNewsPipeline&apos;: 300,# 自定义pipeline视情况选择性注册(可选)</span><br><span class="line">&#125;</span><br><span class="line"># Enable and configure HTTP caching (disabled by default)</span><br><span class="line"># See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># ----------------scrapy默认已经自带了缓存，配置如下-----------------</span><br><span class="line"># 打开缓存</span><br><span class="line">#HTTPCACHE_ENABLED = True</span><br><span class="line"># 设置缓存过期时间（单位：秒）</span><br><span class="line">#HTTPCACHE_EXPIRATION_SECS = 0</span><br><span class="line"># 缓存路径(默认为：.scrapy/httpcache)</span><br><span class="line">#HTTPCACHE_DIR = &apos;httpcache&apos;</span><br><span class="line"># 忽略的状态码</span><br><span class="line">#HTTPCACHE_IGNORE_HTTP_CODES = []</span><br><span class="line"># HTTPERROR_ALLOWED_CODES = [302, 301]</span><br><span class="line"># 缓存模式(文件缓存)</span><br><span class="line">#HTTPCACHE_STORAGE = &apos;scrapy.extensions.httpcache.FilesystemCacheStorage&apos;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">#-----------------Scrapy-Redis分布式爬虫相关设置如下--------------------------</span><br><span class="line"># Enables scheduling storing requests queue in redis.</span><br><span class="line">#启用Redis调度存储请求队列，使用Scrapy-Redis的调度器,不再使用scrapy的调度器</span><br><span class="line">SCHEDULER = &quot;scrapy_redis.scheduler.Scheduler&quot;</span><br><span class="line"></span><br><span class="line"># Ensure all spiders share same duplicates filter through redis.</span><br><span class="line">#确保所有的爬虫通过Redis去重，使用Scrapy-Redis的去重组件,不再使用scrapy的去重组件</span><br><span class="line">DUPEFILTER_CLASS = &quot;scrapy_redis.dupefilter.RFPDupeFilter&quot;</span><br><span class="line"></span><br><span class="line"># 默认请求序列化使用的是pickle 但是我们可以更改为其他类似的。PS：这玩意儿2.X的可以用。3.X的不能用</span><br><span class="line"># SCHEDULER_SERIALIZER = &quot;scrapy_redis.picklecompat&quot;</span><br><span class="line"></span><br><span class="line"># 使用优先级调度请求队列 （默认使用），</span><br><span class="line"># 使用Scrapy-Redis的从请求集合中取出请求的方式,三种方式择其一即可:</span><br><span class="line"># 分别按(1)请求的优先级/(2)队列FIFO/(先进先出)(3)栈FILO 取出请求（先进后出）</span><br><span class="line"># SCHEDULER_QUEUE_CLASS = &apos;scrapy_redis.queue.PriorityQueue&apos;</span><br><span class="line"># 可选用的其它队列</span><br><span class="line">SCHEDULER_QUEUE_CLASS = &apos;scrapy_redis.queue.FifoQueue&apos;</span><br><span class="line"># SCHEDULER_QUEUE_CLASS = &apos;scrapy_redis.queue.LifoQueue&apos;</span><br><span class="line"></span><br><span class="line"># Don&apos;t cleanup redis queues, allows to pause/resume crawls.</span><br><span class="line">#不清除Redis队列、这样可以暂停/恢复 爬取，</span><br><span class="line"># 允许暂停,redis请求记录不会丢失(重启爬虫不会重头爬取已爬过的页面)</span><br><span class="line">#SCHEDULER_PERSIST = True</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">#----------------------redis的地址配置-------------------------------------</span><br><span class="line"># Specify the full Redis URL for connecting (optional).</span><br><span class="line"># If set, this takes precedence over the REDIS_HOST and REDIS_PORT settings.</span><br><span class="line"># 指定用于连接redis的URL（可选）</span><br><span class="line"># 如果设置此项，则此项优先级高于设置的REDIS_HOST 和 REDIS_PORT</span><br><span class="line"># REDIS_URL = &apos;redis://root:密码@主机ＩＰ:端口&apos;</span><br><span class="line"># REDIS_URL = &apos;redis://root:123456@192.168.8.30:6379&apos;</span><br><span class="line">REDIS_URL = &apos;redis://root:%s@%s:%s&apos;%(password_redis,host_redis,port_redis)</span><br><span class="line"># 自定义的redis参数（连接超时之类的）</span><br><span class="line">REDIS_PARAMS=&#123;&apos;db&apos;: db_redis&#125;</span><br><span class="line"># Specify the host and port to use when connecting to Redis (optional).</span><br><span class="line"># 指定连接到redis时使用的端口和地址（可选）</span><br><span class="line">#REDIS_HOST = &apos;127.0.0.1&apos;</span><br><span class="line">#REDIS_PORT = 6379</span><br><span class="line">#REDIS_PASS = &apos;19940225&apos;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">#-----------------------------------------暂时用不到-------------------------------------------------------</span><br><span class="line"># 它定义了将被允许抓取的网址的长度为URL的最大极限，默认值：2083</span><br><span class="line"># URLLENGTH_LIMIT=2083</span><br><span class="line"># 爬取网站最大允许的深度(depth)值,默认值0。如果为0，则没有限制</span><br><span class="line"># DEPTH_LIMIT = 3</span><br><span class="line"># 整数值。用于根据深度调整request优先级。如果为0，则不根据深度进行优先级调整。</span><br><span class="line"># DEPTH_PRIORITY=3</span><br><span class="line"></span><br><span class="line"># 最大空闲时间防止分布式爬虫因为等待而关闭</span><br><span class="line"># 这只有当上面设置的队列类是SpiderQueue或SpiderStack时才有效</span><br><span class="line"># 并且当您的蜘蛛首次启动时，也可能会阻止同一时间启动（由于队列为空）</span><br><span class="line"># SCHEDULER_IDLE_BEFORE_CLOSE = 10</span><br><span class="line"></span><br><span class="line"># 序列化项目管道作为redis Key存储</span><br><span class="line"># REDIS_ITEMS_KEY = &apos;%(spider)s:items&apos;</span><br><span class="line"></span><br><span class="line"># 默认使用ScrapyJSONEncoder进行项目序列化</span><br><span class="line"># You can use any importable path to a callable object.</span><br><span class="line"># REDIS_ITEMS_SERIALIZER = &apos;json.dumps&apos;</span><br><span class="line"></span><br><span class="line"># 自定义redis客户端类</span><br><span class="line"># REDIS_PARAMS[&apos;redis_cls&apos;] = &apos;myproject.RedisClient&apos;</span><br><span class="line"></span><br><span class="line"># 如果为True，则使用redis的&apos;spop&apos;进行操作。</span><br><span class="line"># 如果需要避免起始网址列表出现重复，这个选项非常有用。开启此选项urls必须通过sadd添加，否则会出现类型错误。</span><br><span class="line"># REDIS_START_URLS_AS_SET = False</span><br><span class="line"></span><br><span class="line"># RedisSpider和RedisCrawlSpider默认 start_usls 键</span><br><span class="line"># REDIS_START_URLS_KEY = &apos;%(name)s:start_urls&apos;</span><br><span class="line"></span><br><span class="line"># 设置redis使用utf-8之外的编码</span><br><span class="line"># REDIS_ENCODING = &apos;latin1&apos;</span><br><span class="line"></span><br><span class="line"># Disable Telnet Console (enabled by default)</span><br><span class="line"># 它定义是否启用telnetconsole,默认值：True</span><br><span class="line">#TELNETCONSOLE_ENABLED = False</span><br><span class="line"></span><br><span class="line"># Enable and configure the AutoThrottle extension (disabled by default)</span><br><span class="line"># See http://doc.scrapy.org/en/latest/topics/autothrottle.html</span><br><span class="line">#AUTOTHROTTLE_ENABLED = True</span><br><span class="line"># The initial download delay</span><br><span class="line"># 开始下载时限速并延迟时间</span><br><span class="line">#AUTOTHROTTLE_START_DELAY = 5</span><br><span class="line"># The maximum download delay to be set in case of high latencies</span><br><span class="line">#高并发请求时最大延迟时间</span><br><span class="line">#AUTOTHROTTLE_MAX_DELAY = 60</span><br><span class="line"># The average number of requests Scrapy should be sending in parallel to</span><br><span class="line"># each remote server</span><br><span class="line">#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0</span><br><span class="line"># Enable showing throttling stats for every response received:</span><br><span class="line">#AUTOTHROTTLE_DEBUG = False</span><br><span class="line"></span><br><span class="line">最后再来两条启用调试的命令：</span><br><span class="line">启用调试工具</span><br><span class="line">命令行调试</span><br><span class="line">scrapy shell url 查看网页，不过这种方式可能对于要请求头的网页不行，对于一般的网页还是可以的</span><br><span class="line">scrapy view shell 用来查看动态加载的网页，如果查看的网页用了动态加载，那么用这个命令行打开的网页就是不完整的，肯定缺少了什么</span><br></pre></td></tr></table></figure>

<p>暂停和恢复爬虫<br>初学者最头疼的事情就是没有处理好异常，当爬虫爬到一半的时候突然因为错误而中断了，但是这时又不能从中断的地方开始继续爬，顿时感觉心里日了狗，但是这里有一个方法可以暂时的存储你爬的状态，当爬虫中断的时候继续打开后依然可以从中断的地方爬，不过虽说持久化可以有效的处理，但是要注意的是当使用cookie临时的模拟登录状态的时候要注意cookie的有效期</p>
<p>只需要在setting.py中JOB_DIR=file_name 其中填的是你的文件目录，注意这里的目录不允许共享，只能存储单独的一个spdire的运行状态，如果你不想在从中断的地方开始运行，只需要将这个文件夹删除即可</p>
<p>当然还有其他的放法：scrapy crawl somespider -s JOBDIR=crawls/somespider-1，这个是在终端启动爬虫的时候调用的，可以通过ctr+c中断，恢复还是输入上面的命令</p>

    </div>

    

    
        <div class="post-tags">
            <i class="fa fa-tags" aria-hidden="true"></i>
            <a href="/blog/tags/爬虫/">#爬虫</a> <a href="/blog/tags/爬虫settings配置/">#爬虫settings配置</a>
        </div>
    

    <!-- Comments -->
    

</div>
        </section>

    </div>
</div>


</div>

<!-- Footer -->
<div class="push"></div>

<footer class="footer-content">
    <div class="container">
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-6 col-lg-6 footer-about">
                <h2>About</h2>
                <p>
                    This theme was developed by <a href="https://github.com/klugjo">Jonathan Klughertz</a>. The source code is available on Github. Create Websites. Make Magic.
                </p>
            </div>
            
    <div class="col-xs-6 col-sm-6 col-md-3 col-lg-3 recent-posts">
        <h2>Recent Posts</h2>
        <ul>
            
            <li>
                <a class="footer-post" href="/blog/2019/10/25/tars框架环境基础搭建/">tars框架环境基础搭建</a>
            </li>
            
            <li>
                <a class="footer-post" href="/blog/2019/10/20/认识tars框架/">认识tars框架</a>
            </li>
            
            <li>
                <a class="footer-post" href="/blog/2019/10/05/redis数据迁移备份与恢复/">redis数据迁移备份与恢复</a>
            </li>
            
            <li>
                <a class="footer-post" href="/blog/2019/09/21/负载均衡的5种策略/">负载均衡的5种策略</a>
            </li>
            
        </ul>
    </div>



            
        </div>
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12">
                <ul class="list-inline footer-social-icons">
                    
                    <li class="list-inline-item">
                        <a href="https://github.com/klugjo/hexo-theme-alpha-dust">
                            <span class="footer-icon-container">
                                <i class="fa fa-github"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://twitter.com/?lang=en">
                            <span class="footer-icon-container">
                                <i class="fa fa-twitter"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.facebook.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-facebook"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.instagram.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-instagram"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://dribbble.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-dribbble"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://plus.google.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-google-plus"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.behance.net/">
                            <span class="footer-icon-container">
                                <i class="fa fa-behance"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://500px.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-500px"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="mailto:test@example.com">
                            <span class="footer-icon-container">
                                <i class="fa fa-envelope-o"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="\#">
                            <span class="footer-icon-container">
                                <i class="fa fa-rss"></i>
                            </span>
                        </a>
                    </li>
                    
                </ul>
            </div>
        </div>
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12">
                <div class="footer-copyright">
                    @Untitled. All right reserved | Design & Hexo <a href="http://www.codeblocq.com/">Jonathan Klughertz</a>
                </div>
            </div>
        </div>
    </div>
</footer>

<!-- After footer scripts -->

<!-- jQuery -->
<script src="//code.jquery.com/jquery-2.1.4.min.js"></script>

<!-- Tween Max -->
<script src="//cdnjs.cloudflare.com/ajax/libs/gsap/1.18.5/TweenMax.min.js"></script>

<!-- Gallery -->
<script src="//cdnjs.cloudflare.com/ajax/libs/featherlight/1.3.5/featherlight.min.js" type="text/javascript" charset="utf-8"></script>

<!-- Custom JavaScript -->
<script src="/blog/js/main.js"></script>

<!-- Disqus Comments -->



</body>

</html>